package cn.net.withub.dataCollector.web.collector;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.net.withub.dataCollector.common.editor.DateUtil;
import cn.net.withub.dataCollector.common.model.*;
import cn.net.withub.dataCollector.common.utils.CreateNewKey;
import cn.net.withub.dataCollector.common.utils.HTMLSpirit;
import cn.net.withub.dataCollector.web.service.CollectorService;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.context.WebApplicationContext;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

/**
 * 四中院
 * Created by hupeng on 2018/6/14.
 */
public class CollectorUtilsM40 extends BreadthCrawler {

    //@Resource
    private CollectorService collectorService;

    //@Resource
    private WebApplicationContext webAppliction;

    private TCollectorConfig tCollectorConfig;

    private Logger log = Logger.getLogger(this.getClass());

    public void CollectorUtils(CollectorService collectorService){
        collectorService = collectorService;
    }

    public CollectorUtilsM40(String crawlPath, boolean autoParse, WebApplicationContext wac, TCollectorConfig config) {
        super(crawlPath, autoParse);

        webAppliction = wac;
        if(webAppliction !=null){
            collectorService = (CollectorService)webAppliction.getBean("collectorService");
        }
        if(config != null){
            tCollectorConfig = config;
        }
    }

    public void visit(Page page, CrawlDatums crawlDatums) {
        //Config.TIMEOUT_CONNECT = 20000;
        /*读取超时*/
        //Config.TIMEOUT_READ = 20000;
        String url = page.getUrl();
        String html = page.getHtml();
        String contentType = page.getResponse().getContentType();
        //log.info("--------------------------URL:\n" + url+"     "+contentType);
        try{
            //http://149.16.0.202/class.asp?classid=326
            //if(page.matchUrl("http://.*/class.asp\\?classid=.*")){
            if(page.matchUrl("http://.*/infolist.asp\\?p=.*&classname=.*&classid=.*") || page.matchUrl("http://.*/infolist.asp\\?classid=.*") || page.matchUrl("http://.*/infolist.asp\\?classname=.*&classid=.*") || page.matchUrl("http://.*/infolist.asp\\?p=.*&classid=.*")){
                //log.info("--------------------------html:\n"+page.html());
                //获取新闻列表内容
                Elements trElements = page.select("table[width=95%][background=images2011/t61.gif][class=sc07] tr:has(td[width=82%][height=35][class=sc07])");
                //log.info("liElements:"+liElements);
                for (Element tr : trElements) {
                    String fbsj = "";
                    String title = "";
                    String src = "";
                    try{
                        //获取发布时间
                        Elements dateElements = tr.select("td[width=15%]");
                        if(dateElements != null && dateElements.size()>0){
                            fbsj = dateElements.get(0).text().replaceAll("  ","").replaceAll(" ","");
                        }
                        fbsj = fbsj.replace(".","-");
                        log.info("fbsj...."+fbsj);

                        //获取发布标题
                        Elements titleElements = tr.select("td[width=82%] a[href][target=_blank][title]");
                        if(titleElements != null && titleElements.size()>0){
                            title = titleElements.attr("title");
                            if(null == title || "".equals(title))
                                title = titleElements.get(0).text();
                        }
                        log.info("title...."+title);

                        Elements elements = tr.select("td[width=82%] a[href][target=_blank][title]");
                        log.info("-----------elements:"+elements+".........");
                        for (Element element : elements) {
                            //a标签 href
                            src = element.attr("abs:href");
                            log.info("href url:\n"+src);
                            if(tCollectorConfig.getFbsj()==null || (tCollectorConfig.getFbsj()!=null && DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(tCollectorConfig.getFbsj()))>0)){
                                log.info("fbsj: "+fbsj);
                                crawlDatums.add(
                                        new CrawlDatum(src)
                                                .meta("title",title)
                                                .meta("fbsj",fbsj)
                                                .meta("firstUrl",url)
                                );
                            }
                            //判断已抓取过的最大发布日期，根据标题和发布时间进行对比，未抓取过的进行抓取
                            else if(tCollectorConfig.getFbsj()!=null && DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(tCollectorConfig.getFbsj()))==0){
                                log.info("fbsj: "+fbsj+"相同,跳过不抓取");
                                TCollectorData tCollectorData = (TCollectorData)collectorService.load(TCollectorData.class," fbsj='"+fbsj+"' and title='"+title+"'");
                                if(tCollectorData == null){
                                    log.info("fbsj: "+fbsj+"抓取");
                                    crawlDatums.add(
                                            new CrawlDatum(src)
                                                    .meta("title",title)
                                                    .meta("fbsj",fbsj)
                                                    .meta("firstUrl",url)
                                    );
                                }
                            }

                        }
                    }catch (Exception e){
                        saveLog(title,fbsj,url,src,0);
                        e.printStackTrace();
                        continue;
                    }
                }

                //获取下一页
                String src = "";
                try{
                    Elements pageElements = page.select("a[class=white_bg]");
                    for(Element element : pageElements){
                        src = element.attr("abs:href");
                        String text = element.text().trim();
                        if(text.equals("下一页")){
                            log.info("抓取下一页......"+src);
                            crawlDatums.add(
                                    new CrawlDatum(src)
                            );
                        }
                    }
                }catch (Exception e){
                    saveLog("","",url,src,0);
                    e.printStackTrace();
                }
            } else if(page.matchUrl("http://.*/view.asp\\?NewsID=.*")){
                //http://149.16.0.202/view.asp?NewsID=335421
                log.info("contentType:     \n"+contentType+"     meta:"+page.meta("title"));
                Map<String,Object> map = new HashMap<String, Object>();
                String content = "";
                String serverFilePath = "";
                String wjgs = "";

                String title = page.meta("title");
                String fbsj = page.meta("fbsj").replaceAll("  ","").replaceAll(" ","").replace(".","-");
                String firstUrl = page.meta("firstUrl");

                int zt = 0;
                log.info("contentType:"+contentType);

                if(contentType.contains("msword") || contentType.contains("pdf")){
                    try{
                        //获取word文件
                        if(contentType.contains("msword")){
                            map = HttpClientUtils.getWord(url);
                            wjgs = "doc";
                        }
                        //获取pdf文件
                        else if(contentType.contains("pdf")){
                            map = HttpClientUtils.getPdf(url);
                            wjgs = "pdf";
                        }

                        if(map != null){
                            content = map.get("content")==null?"":map.get("content").toString();
                            serverFilePath = map.get("serverFilePath")==null?"":map.get("serverFilePath").toString();
                        }

                        Date date = new Date();
                        String id = new CreateNewKey().createId();
                        TCollectorData tCollectorData = new TCollectorData();
                        tCollectorData.setId(id);
                        tCollectorData.setConfigId(tCollectorConfig.getId());
                        tCollectorData.setTitle(title);
                        Calendar nowdate = Calendar.getInstance();
                        String year = String.valueOf(nowdate.get(Calendar.YEAR));
                        if(null != fbsj && !"".equals(fbsj)) {
                            if (Integer.parseInt(fbsj.substring(0, 4)) > Integer.parseInt(year)) {
                                fbsj = year + fbsj.substring(4);
                                //log.info("fbsj:" + fbsj);
                            }
                        }
                        tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        tCollectorData.setWjgs(wjgs);
                        tCollectorData.setUrl(url);
                        tCollectorData.setFydm(tCollectorConfig.getFydm());
                        tCollectorData.setFymc(tCollectorConfig.getFymc());
                        tCollectorData.setWjlj(serverFilePath);
                        tCollectorData.setSystime(date);
                        collectorService.save(tCollectorData);

                        TCollectorContent tCollectorContent = new TCollectorContent();
                        tCollectorContent.setId(new CreateNewKey().createId());
                        tCollectorContent.setContent(content);
                        tCollectorContent.setSystime(date);
                        tCollectorContent.setDataId(id);
                        collectorService.save(tCollectorContent);

                        //保存最大发布时间
                        TCollectorConfig config = (TCollectorConfig)collectorService.load(TCollectorConfig.class," id='"+tCollectorConfig.getId()+"'");
                        if(config.getFbsj()==null || DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj()))>0){
                            log.info("config........."+fbsj);
                            config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                            collectorService.save(config);
                        }

                        zt = 1;
                    }catch(Exception e){
                        zt = 0;
                        e.printStackTrace();
                    }finally {
                        saveLog(title,fbsj,firstUrl,url,zt);
                    }
                } else if(contentType.contains("html")){//html文件
                    try{
                        //获取iframe内容
                        Elements iframe=page.select("iframe[id=ifrName][name=ifrName]");
                        String src = iframe.attr("abs:src");
                        crawlDatums.add(new CrawlDatum(src)
                                .meta("title",title)
                                .meta("fbsj",fbsj)
                                .meta("firstUrl",firstUrl)
                                .meta("tqUrl",url)
                        );
                    }catch (Exception e){
                        saveLog(title,fbsj,firstUrl,url,0);
                        e.printStackTrace();
                    }
                }
            }else if(page.matchUrl("http://.*/displaycont.asp\\?newsid=.*&pinglun=.*&classname1=.*") || page.matchUrl("http://.*/displaycont.asp\\?newsid=.*&pinglun=.*") ||page.matchUrl("http://.*/displaycont.asp\\?newsid=.*") || page.matchUrl("http://.*/displaycont.asp\\?newsid=.*&classname1=.*")){
                //displaycont.asp?newsid=335292&pinglun=1&classname1=每日必读
                //保存爬取信息
                String title = page.meta("title")==null?"":page.meta("title");
                String fbsj = page.meta("fbsj")==null?"":page.meta("fbsj").replace(".","-");
                String firstUrl = page.meta("firstUrl")==null?"":page.meta("firstUrl");
                String tqUrl = page.meta("tqUrl")==null?"":page.meta("tqUrl");
                int zt = 0;

                try{
                    String ttt1 = page.select("td[height=35][colspan=3][background=images2011/t61.gif][class=zc04]  span[class=sc10]").text();
                    log.info("---------------ttt1:"+ttt1);//作者
                    //发表人：szfyyjs　发布时间：2018.05.30 点击次数：28 次
                    int index_fbsj = ttt1.indexOf("发布时间：");
                    int index_bjsh = ttt1.indexOf("发表人：");
                    int index_djcs = ttt1.indexOf("点击次数：");

                    if (index_fbsj > -1 && (null == fbsj || "".equals(fbsj.trim()))) {
                        fbsj = ttt1.substring(index_fbsj + 5, index_djcs).replaceAll(" ", "").replaceAll("  ", "").replace(".", "-");
                    }
                    String bjsh = null;
                    if (index_bjsh > -1) {
                            bjsh = ttt1.substring(index_bjsh + 4, index_fbsj);
                    }
                    String djcs = null;
                    if (index_djcs > -1)
                        djcs = ttt1.substring(index_djcs + 5);
                    //如果是pdf、doc
                    Elements elements = page.select("td[style=line-height:28px;font-size:14px;] span[class=c11] a[href][target=_blank]");
                    if(null != elements && elements.size() > 0){
                        try{
                            for (Element e : elements) {
                                String href = e.attr("abs:href");
                                if(null != href && (href.trim().endsWith(".doc") || href.trim().endsWith(".pdf"))){
                                    crawlDatums.add(new CrawlDatum(href)
                                            .meta("title",title)
                                            .meta("fbsj",fbsj)
                                            .meta("firstUrl",firstUrl)
                                            .meta("tqUrl",url)
                                            .meta("bjsh",bjsh)
                                            .meta("djcs",djcs)
                                    );
                                }

                            }
                        }catch (Exception e){
                            saveLog(title,fbsj,firstUrl,tqUrl,0);
                            e.printStackTrace();
                        }

                    }else{//html
                        try {
                            page.select("td[width=30%][height=35][class=sc09][style=padding-left:10px]:has(strong)").remove();
                            html = page.select("td[style=line-height:28px;font-size:14px;]:has(span[class=c11])").html();
                            Date date = new Date();
                            String id = new CreateNewKey().createId();
                            TCollectorData tCollectorData = new TCollectorData();
                            tCollectorData.setId(id);
                            tCollectorData.setConfigId(tCollectorConfig.getId());
                            tCollectorData.setTitle(title);
                            Calendar nowdate = Calendar.getInstance();
                            String year = String.valueOf(nowdate.get(Calendar.YEAR));
                            if(null != fbsj && !"".equals(fbsj)) {
                                if (Integer.parseInt(fbsj.substring(0, 4)) > Integer.parseInt(year)) {
                                    fbsj = year + fbsj.substring(4);
                                    //log.info("fbsj:" + fbsj);
                                }
                            }
                            tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                            tCollectorData.setXxly("");
                            tCollectorData.setXxfb("");
                            tCollectorData.setBjsh(bjsh);
                            tCollectorData.setXxzz(bjsh);
                            tCollectorData.setDjcs(djcs);
                            tCollectorData.setWjgs("html");
                            tCollectorData.setUrl(tqUrl);
                            tCollectorData.setFydm(tCollectorConfig.getFydm());
                            tCollectorData.setFymc(tCollectorConfig.getFymc());
                            tCollectorData.setSystime(date);
                            collectorService.save(tCollectorData);

                            TCollectorContent tCollectorContent = new TCollectorContent();
                            tCollectorContent.setId(new CreateNewKey().createId());
                            tCollectorContent.setContent(html);
                            tCollectorContent.setText(HTMLSpirit.getTextFromTHML(html));
                            tCollectorContent.setSystime(date);
                            tCollectorContent.setDataId(id);
                            collectorService.save(tCollectorContent);

                            //遍历所有html中所有图片
                            Elements imgs = page.select("img[src]");
                            if(null != imgs && imgs.size() > 0){
                                for (Element img : imgs) {
                                    String src = img.attr("abs:src").replace("../","");
                                    log.info("img url:\n"+src);
                                    crawlDatums.add(new CrawlDatum(src)
                                            .meta("dataId",id)
                                    );
                                }
                            }

                            //保存最大发布时间
                            TCollectorConfig config = (TCollectorConfig)collectorService.load(TCollectorConfig.class," id='"+tCollectorConfig.getId()+"'");
                            if(config.getFbsj()==null || DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj()))>0){
                                config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                                collectorService.save(config);
                            }

                            zt = 1;

                        }catch (Exception e){
                            zt = 0;
                            e.printStackTrace();
                        }finally {
                            saveLog(title,fbsj,firstUrl,tqUrl,zt);
                        }

                    }

                }catch(Exception e){
                    saveLog(title,fbsj,firstUrl,tqUrl,0);
                    e.printStackTrace();
                }
            }else if(page.matchUrl("http://.*/upfiles/.*.doc") || page.matchUrl("http://.*/upfiles/.*.pdf")) {
                Map<String,Object> map = new HashMap<String, Object>();
                String content = "";
                String serverFilePath = "";
                String wjgs = "";

                String title = page.meta("title");
                String fbsj = page.meta("fbsj").replaceAll("  ","").replaceAll(" ","").replace(".","-");
                String firstUrl = page.meta("firstUrl");
                String bjsh = page.meta("bjsh");
                String djcs = page.meta("djcs");
                int zt = 0;
                if(contentType.contains("msword") || contentType.contains("pdf")){
                    try{
                        //获取word文件
                        if(contentType.contains("msword")){
                            map = HttpClientUtils.getWord(url);
                            wjgs = "doc";
                        }
                        //获取pdf文件
                        else if(contentType.contains("pdf")){
                            map = HttpClientUtils.getPdf(url);
                            wjgs = "pdf";
                        }
                        if(map != null){
                            content = map.get("content")==null?"":map.get("content").toString();
                            serverFilePath = map.get("serverFilePath")==null?"":map.get("serverFilePath").toString();
                        }

                        Date date = new Date();
                        String id = new CreateNewKey().createId();
                        TCollectorData tCollectorData = new TCollectorData();
                        tCollectorData.setId(id);
                        tCollectorData.setConfigId(tCollectorConfig.getId());
                        tCollectorData.setTitle(title);
                        Calendar nowdate = Calendar.getInstance();
                        String year = String.valueOf(nowdate.get(Calendar.YEAR));
                        if(null != fbsj && !"".equals(fbsj)) {
                            if (Integer.parseInt(fbsj.substring(0, 4)) > Integer.parseInt(year)) {
                                fbsj = year + fbsj.substring(4);
                                //log.info("fbsj:" + fbsj);
                            }
                        }
                        tCollectorData.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                        tCollectorData.setWjgs(wjgs);
                        tCollectorData.setUrl(url);
                        tCollectorData.setBjsh(bjsh);
                        tCollectorData.setDjcs(djcs);
                        tCollectorData.setFydm(tCollectorConfig.getFydm());
                        tCollectorData.setFymc(tCollectorConfig.getFymc());
                        tCollectorData.setWjlj(serverFilePath);
                        tCollectorData.setSystime(date);
                        collectorService.save(tCollectorData);

                        TCollectorContent tCollectorContent = new TCollectorContent();
                        tCollectorContent.setId(new CreateNewKey().createId());
                        tCollectorContent.setContent(content);
                        tCollectorContent.setSystime(date);
                        tCollectorContent.setDataId(id);
                        collectorService.save(tCollectorContent);

                        //保存最大发布时间
                        TCollectorConfig config = (TCollectorConfig)collectorService.load(TCollectorConfig.class," id='"+tCollectorConfig.getId()+"'");
                        if(config.getFbsj()==null || DateUtil.compareDate(fbsj,new SimpleDateFormat("yyyy-MM-dd").format(config.getFbsj()))>0){
                            log.info("-----config........."+fbsj);
                            config.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
                            collectorService.save(config);
                        }

                        zt = 1;
                    }catch(Exception e){
                        zt = 0;
                        e.printStackTrace();
                    }finally {
                        saveLog(title,fbsj,firstUrl,url,zt);
                    }
                }
            } else if(contentType !=null && contentType.startsWith("image")){//html图片
                String dataId = page.meta("dataId")==null?"":page.meta("dataId");
                String imageName = url.substring(url.lastIndexOf("/")+1,url.length());
                //log.info("-----imageName:"+imageName);
                String serverFilePath = "";
                try {
                    serverFilePath = HttpClientUtils.getImage(url);
                } catch (IOException e) {
                    serverFilePath = "";
                    e.printStackTrace();
                }finally {
                    TCollectorElement tCollectorElement = new TCollectorElement();
                    tCollectorElement.setId(new CreateNewKey().createId());
                    tCollectorElement.setDataId(dataId);
                    tCollectorElement.setTitle(imageName);
                    tCollectorElement.setUrl(url);
                    tCollectorElement.setWjlj(serverFilePath);
                    tCollectorElement.setSystime(new Date());
                    collectorService.save(tCollectorElement);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }
    public  String getHtml(String httpPath){
        try {
            //建立连接
            URL url = new URL(httpPath);
            HttpURLConnection httpUrlConn = (HttpURLConnection) url.openConnection();
            httpUrlConn.setDoInput(true);
            httpUrlConn.setRequestMethod("GET");
            httpUrlConn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
            //获取输入流
            InputStream input = httpUrlConn.getInputStream();
            //将字节输入流转换为字符输入流
            InputStreamReader read = new InputStreamReader(input, "utf-8");
            //为字符输入流添加缓冲
            BufferedReader br = new BufferedReader(read);
            // 读取返回结果
            String data = br.readLine();
            if (null != data) {
                data = data.replaceAll("[a-zA-Z]","").replaceAll("[\\pP\\pZ\\pS\\pM\\pC]","");
            }
            // 释放资源
            br.close();
            read.close();
            input.close();
            httpUrlConn.disconnect();
            //document.write('338');
            System.out.println("data:"+data);
            return  data;
        } catch (MalformedURLException e) {
            e.printStackTrace();
            return  null;
        } catch (IOException e) {
            e.printStackTrace();
            return  null;
        }
    }

    public void saveLog(String title,String fbsj,String firstUrl,String tqUrl,int zt){
        try {
            log.info("savelog........."+fbsj);
            TCollectorLog tCollectorLog = new TCollectorLog();
            tCollectorLog.setId(new CreateNewKey().createId());
            tCollectorLog.setConfigId(tCollectorConfig.getId());
            tCollectorLog.setIp(tCollectorConfig.getIp());
            tCollectorLog.setModuleName(tCollectorConfig.getModuleName());
            tCollectorLog.setFydm(tCollectorConfig.getFydm());
            tCollectorLog.setFymc(tCollectorConfig.getFymc());
            tCollectorLog.setTitle(title);
            try {
                tCollectorLog.setFbsj(new SimpleDateFormat("yyyy-MM-dd").parse(fbsj));
            }catch (Exception e) {
                e.printStackTrace();
            }
            tCollectorLog.setZt(zt);
            tCollectorLog.setFirstUrl(firstUrl);
            tCollectorLog.setTqUrl(tqUrl);
            tCollectorLog.setSysTime(new Date());

            collectorService.save(tCollectorLog);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) throws Exception {
        //getHtml("http://149.72.0.202/www/js/gethits.asp?id=3969");

    //CollectorUtilsM1F crawler = new CollectorUtilsM1F("crawl", true,null,null);
        //crawler.setThreads(10);
        //crawler.setTopN(100);
       //crawler.setResumable(true);
        //crawler.start(500);

        /*Executor executor=new Executor() {
            public void execute(CrawlDatum datum, CrawlDatums next) throws Exception {
                HtmlUnitDriver driver = new HtmlUnitDriver();
                driver.setJavascriptEnabled(true);
                driver.get(datum.getUrl());
                //WebElement element=driver.findElementByCssSelector("ul#ssxw");
                WebElement element=driver.findElementByCssSelector("span#outlink1");
                log.info("反链数:"+element.getText());
            }
        };

        //创建一个基于伯克利DB的DBManager
        DBManager manager=new BerkeleyDBManager("crawl");
        //创建一个Crawler需要有DBManager和Executor
        Crawler crawler= new Crawler(manager,executor);
        //crawler.addSeed("http://149.0.160.10");
        crawler.addSeed("http://seo.chinaz.com/?host=www.tuicool.com");
        crawler.start(1);*/
    }

}
